{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "# Grad-CAM class activation visualization\n",
    "\n",
    "**Author:** [fchollet](https://twitter.com/fchollet)<br>\n",
    "**Date created:** 2020/04/26<br>\n",
    "**Last modified:** 2020/05/14<br>\n",
    "**Description:** How to obtain a class activation heatmap for an image classification model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "Adapted from Deep Learning with Python (2017).\n",
    "\n",
    "## Setup\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "\n",
    "# Display\n",
    "from IPython.display import Image\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.cm as cm\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Configurable parameters\n",
    "\n",
    "You can change these to another model.\n",
    "\n",
    "To get the values for `last_conv_layer_name` and `classifier_layer_names`, use\n",
    " `model.summary()` to see the names of all layers in the model.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "model_builder = keras.applications.xception.Xception\n",
    "img_size = (299, 299)\n",
    "preprocess_input = keras.applications.xception.preprocess_input\n",
    "decode_predictions = keras.applications.xception.decode_predictions\n",
    "\n",
    "last_conv_layer_name = \"block14_sepconv2_act\"\n",
    "classifier_layer_names = [\n",
    "    \"avg_pool\",\n",
    "    \"predictions\",\n",
    "]\n",
    "\n",
    "# The local path to our target image\n",
    "img_path = keras.utils.get_file(\n",
    "    \"african_elephant.jpg\", \" https://i.imgur.com/Bvro0YD.png\"\n",
    ")\n",
    "\n",
    "display(Image(img_path))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## The Grad-CAM algorithm\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "\n",
    "def get_img_array(img_path, size):\n",
    "    # `img` is a PIL image of size 299x299\n",
    "    img = keras.preprocessing.image.load_img(img_path, target_size=size)\n",
    "    # `array` is a float32 Numpy array of shape (299, 299, 3)\n",
    "    array = keras.preprocessing.image.img_to_array(img)\n",
    "    # We add a dimension to transform our array into a \"batch\"\n",
    "    # of size (1, 299, 299, 3)\n",
    "    array = np.expand_dims(array, axis=0)\n",
    "    return array\n",
    "\n",
    "\n",
    "def make_gradcam_heatmap(\n",
    "    img_array, model, last_conv_layer_name, classifier_layer_names\n",
    "):\n",
    "    # First, we create a model that maps the input image to the activations\n",
    "    # of the last conv layer\n",
    "    last_conv_layer = model.get_layer(last_conv_layer_name)\n",
    "    last_conv_layer_model = keras.Model(model.inputs, last_conv_layer.output)\n",
    "\n",
    "    # Second, we create a model that maps the activations of the last conv\n",
    "    # layer to the final class predictions\n",
    "    classifier_input = keras.Input(shape=last_conv_layer.output.shape[1:])\n",
    "    x = classifier_input\n",
    "    for layer_name in classifier_layer_names:\n",
    "        x = model.get_layer(layer_name)(x)\n",
    "    classifier_model = keras.Model(classifier_input, x)\n",
    "\n",
    "    # Then, we compute the gradient of the top predicted class for our input image\n",
    "    # with respect to the activations of the last conv layer\n",
    "    with tf.GradientTape() as tape:\n",
    "        # Compute activations of the last conv layer and make the tape watch it\n",
    "        last_conv_layer_output = last_conv_layer_model(img_array)\n",
    "        tape.watch(last_conv_layer_output)\n",
    "        # Compute class predictions\n",
    "        preds = classifier_model(last_conv_layer_output)\n",
    "        top_pred_index = tf.argmax(preds[0])\n",
    "        top_class_channel = preds[:, top_pred_index]\n",
    "\n",
    "    # This is the gradient of the top predicted class with regard to\n",
    "    # the output feature map of the last conv layer\n",
    "    grads = tape.gradient(top_class_channel, last_conv_layer_output)\n",
    "\n",
    "    # This is a vector where each entry is the mean intensity of the gradient\n",
    "    # over a specific feature map channel\n",
    "    pooled_grads = tf.reduce_mean(grads, axis=(0, 1, 2))\n",
    "\n",
    "    # We multiply each channel in the feature map array\n",
    "    # by \"how important this channel is\" with regard to the top predicted class\n",
    "    last_conv_layer_output = last_conv_layer_output.numpy()[0]\n",
    "    pooled_grads = pooled_grads.numpy()\n",
    "    for i in range(pooled_grads.shape[-1]):\n",
    "        last_conv_layer_output[:, :, i] *= pooled_grads[i]\n",
    "\n",
    "    # The channel-wise mean of the resulting feature map\n",
    "    # is our heatmap of class activation\n",
    "    heatmap = np.mean(last_conv_layer_output, axis=-1)\n",
    "\n",
    "    # For visualization purpose, we will also normalize the heatmap between 0 & 1\n",
    "    heatmap = np.maximum(heatmap, 0) / np.max(heatmap)\n",
    "    return heatmap\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Let's test-drive it\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "# Prepare image\n",
    "img_array = preprocess_input(get_img_array(img_path, size=img_size))\n",
    "\n",
    "# Make model\n",
    "model = model_builder(weights=\"imagenet\")\n",
    "\n",
    "# Print what the top predicted class is\n",
    "preds = model.predict(img_array)\n",
    "print(\"Predicted:\", decode_predictions(preds, top=1)[0])\n",
    "\n",
    "# Generate class activation heatmap\n",
    "heatmap = make_gradcam_heatmap(\n",
    "    img_array, model, last_conv_layer_name, classifier_layer_names\n",
    ")\n",
    "\n",
    "# Display heatmap\n",
    "plt.matshow(heatmap)\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Create a superimposed visualization\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "# We load the original image\n",
    "img = keras.preprocessing.image.load_img(img_path)\n",
    "img = keras.preprocessing.image.img_to_array(img)\n",
    "\n",
    "# We rescale heatmap to a range 0-255\n",
    "heatmap = np.uint8(255 * heatmap)\n",
    "\n",
    "# We use jet colormap to colorize heatmap\n",
    "jet = cm.get_cmap(\"jet\")\n",
    "\n",
    "# We use RGB values of the colormap\n",
    "jet_colors = jet(np.arange(256))[:, :3]\n",
    "jet_heatmap = jet_colors[heatmap]\n",
    "\n",
    "# We create an image with RGB colorized heatmap\n",
    "jet_heatmap = keras.preprocessing.image.array_to_img(jet_heatmap)\n",
    "jet_heatmap = jet_heatmap.resize((img.shape[1], img.shape[0]))\n",
    "jet_heatmap = keras.preprocessing.image.img_to_array(jet_heatmap)\n",
    "\n",
    "# Superimpose the heatmap on original image\n",
    "superimposed_img = jet_heatmap * 0.4 + img\n",
    "superimposed_img = keras.preprocessing.image.array_to_img(superimposed_img)\n",
    "\n",
    "# Save the superimposed image\n",
    "save_path = \"elephant_cam.jpg\"\n",
    "superimposed_img.save(save_path)\n",
    "\n",
    "# Display Grad CAM\n",
    "display(Image(save_path))\n"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "grad_cam",
   "private_outputs": false,
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}